<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Deep Dive on Doc Values | Elasticsearch: The Definitive Guide [2.x] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch: The Definitive Guide [2.x]">
<link rel="up" href="docvalues-and-fielddata.html" title="Doc Values and Fielddata">
<link rel="prev" href="docvalues.html" title="Doc Values">
<link rel="next" href="aggregations-and-analysis.html" title="Aggregations and Analysis">
<meta name="DC.type" content="Learn/Docs/Legacy/Elasticsearch/Definitive Guide/2.x">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="2.x">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<p>
  <strong>WARNING</strong>: The 2.x versions of Elasticsearch have passed their
  <a href="https://www.elastic.co/support/eol">EOL dates</a>. If you are running
  a 2.x version, we strongly advise you to upgrade.
</p>
<p>
  This documentation is no longer maintained and may be removed. For the latest
  information, see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html">current
  Elasticsearch documentation</a>.
</p>
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch: The Definitive Guide [2.x]</a></span>
»
<span class="breadcrumb-link"><a href="aggregations.html">Aggregations</a></span>
»
<span class="breadcrumb-link"><a href="docvalues-and-fielddata.html">Doc Values and Fielddata</a></span>
»
<span class="breadcrumb-node">Deep Dive on Doc Values</span>
</div>
<div class="navheader">
<span class="prev">
<a href="docvalues.html">« Doc Values</a>
</span>
<span class="next">
<a href="aggregations-and-analysis.html">Aggregations and Analysis »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="_deep_dive_on_doc_values"></a>Deep Dive on Doc Values<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/300_Aggregations/93_technical_docvalues.asciidoc">edit</a>
</h2>
</div></div></div>
<p>The last section opened by saying doc values are <em>"fast, efficient and memory-friendly"</em>.
Those are some nice marketing buzzwords, but how do doc values actually work?</p>
<p>Doc values are generated at index-time, alongside the creation of the inverted index.
That means doc values are generated on a per-segment basis and are immutable, just like
the inverted index used for search. And, like the inverted index, doc values are serialized
to disk.  This is important to performance and scalability.</p>
<p>By serializing a persistent data structure to disk, we can rely on the OS’s file
system cache to manage memory instead of retaining structures on the JVM heap.
In situations where the "working set" of data is smaller than the available
memory, the OS will naturally keep the doc values resident in memory.  This gives
the same performance profile as on-heap data structures.</p>
<p>But when your working set is much larger than available memory, the OS will begin
paging the doc values on/off disk as required.  This will obviously be slower
than an entirely memory-resident data structure, but it has the advantage of scaling
well beyond the server’s memory capacity.  If these data structures were
purely on-heap, the only option is to crash with an OutOfMemory exception (or implement
a paging scheme just like the OS).</p>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>Because doc values are not managed by the JVM, Elasticsearch servers can be
configured with a much smaller heap.  This gives more memory to the OS for caching.
It also has the benefit of letting the JVM’s garbage collector work with a smaller
heap, which will result in faster and more efficient collection cycles.</p>
<p>Traditionally, the recommendation has been to dedicate 50% of the machine’s memory
to the JVM heap.  With the introduction of doc values, this recommendation is starting
to slide.  Consider giving far less to the heap, perhaps 4-16gb on a 64gb machine,
instead of the full 32gb previously recommended.</p>
<p>For a more detailed discussion, see <a class="xref" href="heap-sizing.html" title="Heap: Sizing and Swapping">Heap: Sizing and Swapping</a>.</p>
</div>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_column_store_compression"></a>Column-store compression<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/300_Aggregations/93_technical_docvalues.asciidoc">edit</a>
</h3>
</div></div></div>
<p>At a high level, doc values are essentially a serialized <em>column-store</em>.  As we
discussed in the last section, column-stores excel at certain operations because
the data is naturally laid out in a fashion that is amenable to those queries.</p>
<p>But they also excel at compressing data, particularly numbers.  This is important for both saving space
on disk <em>and</em> for faster access.  Modern CPU’s are many orders of magnitude faster
than disk drives (although the gap is narrowing quickly with upcoming NVMe drives).  That means
it is often advantageous to minimize the amount of data that must be read from disk,
even if it requires extra CPU cycles to decompress.</p>
<p>To see how it can help compression, take this set of doc values for a numeric field:</p>
<pre class="literallayout">Doc      Terms
-----------------------------------------------------------------
Doc_1 | 100
Doc_2 | 1000
Doc_3 | 1500
Doc_4 | 1200
Doc_5 | 300
Doc_6 | 1900
Doc_7 | 4200
-----------------------------------------------------------------</pre>

<p>The column-stride layout means we have a contiguous block of numbers:
<code class="literal">[100,1000,1500,1200,300,1900,4200]</code>.  Because we know they are all numbers
(instead of a heterogeneous collection like you’d see in a document or row)
values can be packed tightly together with uniform offsets.</p>
<p>Further, there are a variety of compression tricks we can apply to these numbers.
You’ll notice that each of the above numbers are a multiple of 100.  Doc values
detect when all the values in a segment share a <em>greatest common divisor</em> and use
that to compress the values further.</p>
<p>If we save <code class="literal">100</code> as the divisor for this segment, we can divide each number by 100
to get:  <code class="literal">[1,10,15,12,3,19,42]</code>.  Now that the numbers are smaller, they require
fewer bits to store and we’ve reduced the size on-disk.</p>
<p>Doc values use several tricks like this.  In order, the following compression
schemes are checked:</p>
<div class="olist orderedlist">
<ol class="orderedlist">
<li class="listitem">
If all values are identical (or missing), set a flag and record the value
</li>
<li class="listitem">
If there are fewer than 256 values, a simple table encoding is used
</li>
<li class="listitem">
If there are &gt; 256 values, check to see if there is a common divisor
</li>
<li class="listitem">
If there is no common divisor, encode everything as an offset from the smallest
value
</li>
</ol>
</div>
<p>You’ll note that these compression schemes are not "traditional" general purpose
compression like DEFLATE or LZ4.  Because the structure of column-stores are
rigid and well-defined, we can achieve higher compression by using specialized
schemes rather than the more general compression algorithms like LZ4.</p>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>You may be thinking <em>"Well that’s great for numbers, but what about strings?"</em>
Strings are encoded similarly, with the help of an ordinal table.  The
strings are de-duplicated and sorted into a table, assigned an ID, and then those
ID’s are used as numeric doc values.  Which means strings enjoy many of the same
compression benefits that numerics do.</p>
<p>The ordinal table itself has some compression tricks, such as using fixed, variable
or prefix-encoded strings.</p>
</div>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_disabling_doc_values"></a>Disabling Doc Values<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/300_Aggregations/93_technical_docvalues.asciidoc">edit</a>
</h3>
</div></div></div>
<p>Doc values are enabled by default for all fields <em>except</em> analyzed strings.  That means
all numerics, geo_points, dates, IPs and <code class="literal">not_analyzed</code> strings.</p>
<p>Analyzed strings are not able to use doc values at this time; the analysis process
generates many tokens and does not work efficiently with doc values.  We’ll discuss
using analyzed strings for aggregations in <a class="xref" href="aggregations-and-analysis.html" title="Aggregations and Analysis">Aggregations and Analysis</a>.</p>
<p>Because doc values are on by default, you have the option to aggregate and sort
on most fields in your dataset.  But what if you know you will <em>never</em> aggregate,
sort or script on a certain field?</p>
<p>While rare, these circumstances do arise and you may wish to disable doc values
on that particular field.  This will save you some disk space (since the doc values
are not being serialized to disk anymore) and may increase indexing speed slightly
(since the doc values don’t need to be generated).</p>
<p>To disable doc values, set <code class="literal">doc_values: false</code> in the field’s mapping.  For example,
here we create a new index where doc values are disabled for the <code class="literal">"session_id"</code> field:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">PUT my_index
{
  "mappings": {
    "my_type": {
      "properties": {
        "session_id": {
          "type":       "string",
          "index":      "not_analyzed",
          "doc_values": false <a id="CO216-1"></a><i class="conum" data-value="1"></i>
        }
      }
    }
  }
}</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO216-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>By setting <code class="literal">doc_values: false</code>, this field will not be usable in aggregations, sorts
or scripts</p>
</td>
</tr>
</table>
</div>
<p>It is possible to configure the inverse relationship too: make a field available
for aggregations via doc values, but make it unavailable for normal search by disabling
the inverted index.  For example:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">PUT my_index
{
  "mappings": {
    "my_type": {
      "properties": {
        "customer_token": {
          "type":       "string",
          "index":      "not_analyzed",
          "doc_values": true, <a id="CO217-1"></a><i class="conum" data-value="1"></i>
          "index": "no" <a id="CO217-2"></a><i class="conum" data-value="2"></i>
        }
      }
    }
  }
}</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO217-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Doc values are enabled to allow aggregations</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO217-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>Indexing is disabled, which makes the field unavailable to queries/searches</p>
</td>
</tr>
</table>
</div>
<p>By setting <code class="literal">doc_values: true</code> and <code class="literal">index: no</code>, we generate a field which can <em>only</em>
be used in aggregations/sorts/scripts.  This is admittedly a very rare requirement,
but sometimes useful.</p>
</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="docvalues.html">« Doc Values</a>
</span>
<span class="next">
<a href="aggregations-and-analysis.html">Aggregations and Analysis »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
